home *** CD-ROM | disk | FTP | other *** search
/ Clickx 47 / Clickx 47.iso / assets / software / Miro_Installer.exe / Miro_Downloader.exe / downloader.pyc (.txt) < prev    next >
Encoding:
Python Compiled Bytecode  |  2008-01-10  |  20.4 KB  |  700 lines
  1. # Source Generated with Decompyle++
  2. # File: in.pyc (Python 2.5)
  3.  
  4. from base64 import b64encode
  5. from gtcache import gettext as _
  6. from threading import RLock
  7. import os
  8. import re
  9. import shutil
  10. from database import DDBObject, defaultDatabase
  11. from dl_daemon import daemon, command
  12. from download_utils import nextFreeFilename, getFileURLPath, filterDirectoryName
  13. from util import getTorrentInfoHash, returnsUnicode, checkU, returnsFilename, unicodify, checkF, stringify
  14. from platformutils import FilenameType
  15. import app
  16. import config
  17. import httpclient
  18. import indexes
  19. import prefs
  20. import random
  21. import views
  22. import platformutils
  23. import flashscraper
  24. import logging
  25. import traceback
  26. import templatehelper
  27. import fileutil
  28. _downloads = { }
  29.  
  30. def findHTTPAuth(host, path, realm = None, scheme = None):
  31.     checkU(host)
  32.     checkU(path)
  33.     if realm:
  34.         checkU(realm)
  35.     
  36.     if scheme:
  37.         checkU(scheme)
  38.     
  39.     defaultDatabase.confirmDBThread()
  40.     for obj in views.httpauths:
  41.         if obj.host == host and path.startswith(obj.path):
  42.             if realm is None or obj.realm == realm:
  43.                 if scheme is None or obj.authScheme == scheme:
  44.                     return obj
  45.                     continue
  46.     
  47.  
  48.  
  49. class HTTPAuthPassword(DDBObject):
  50.     
  51.     def __init__(self, username, password, host, realm, path, authScheme = u'Basic'):
  52.         checkU(username)
  53.         checkU(password)
  54.         checkU(host)
  55.         checkU(realm)
  56.         checkU(path)
  57.         checkU(authScheme)
  58.         oldAuth = findHTTPAuth(host, path, realm, authScheme)
  59.         while oldAuth is not None:
  60.             oldAuth.remove()
  61.             oldAuth = findHTTPAuth(host, path, realm, authScheme)
  62.         self.username = username
  63.         self.password = password
  64.         self.host = host
  65.         self.realm = realm
  66.         self.path = os.path.dirname(path)
  67.         self.authScheme = authScheme
  68.         DDBObject.__init__(self)
  69.  
  70.     
  71.     def getAuthToken(self):
  72.         authString = u':'
  73.         self.confirmDBThread()
  74.         authString = self.username + u':' + self.password
  75.         return b64encode(authString)
  76.  
  77.     
  78.     def getAuthScheme(self):
  79.         self.confirmDBThread()
  80.         return self.authScheme
  81.  
  82.  
  83. totalUpRate = 0
  84. totalDownRate = 0
  85.  
  86. def _getDownloader(dlid):
  87.     return views.remoteDownloads.getItemWithIndex(indexes.downloadsByDLID, dlid)
  88.  
  89.  
  90. def generateDownloadID():
  91.     dlid = u'download%08d' % random.randint(0, 99999999)
  92.     while _getDownloader(dlid = dlid):
  93.         dlid = u'download%08d' % random.randint(0, 99999999)
  94.     return dlid
  95.  
  96. generateDownloadID = returnsUnicode(generateDownloadID)
  97.  
  98. class RemoteDownloader(DDBObject):
  99.     '''Download a file using the downloader daemon.'''
  100.     
  101.     def __init__(self, url, item, contentType = None, channelName = None):
  102.         checkU(url)
  103.         if contentType:
  104.             checkU(contentType)
  105.         
  106.         self.origURL = self.url = url
  107.         self.itemList = [
  108.             item]
  109.         self.dlid = generateDownloadID()
  110.         self.status = { }
  111.         if contentType is None:
  112.             enclosureContentType = item.getFirstVideoEnclosureType()
  113.             if enclosureContentType == u'application/x-bittorrent':
  114.                 contentType = enclosureContentType
  115.             
  116.         
  117.         self.contentType = u''
  118.         self.deleteFiles = True
  119.         self.channelName = channelName
  120.         self.manualUpload = False
  121.         DDBObject.__init__(self)
  122.         if contentType is None:
  123.             self.contentType = u''
  124.         else:
  125.             self.contentType = contentType
  126.         if self.contentType == u'':
  127.             self.getContentType()
  128.         else:
  129.             self.runDownloader()
  130.  
  131.     
  132.     def signalChange(self, needsSave = True, needsSignalItem = True):
  133.         if needsSignalItem:
  134.             for item in self.itemList:
  135.                 item.signalChange(needsSave = False)
  136.             
  137.         
  138.         DDBObject.signalChange(self, needsSave = needsSave)
  139.  
  140.     
  141.     def onContentType(self, info):
  142.         if not self.idExists():
  143.             return None
  144.         
  145.         if info['status'] == 200:
  146.             self.url = info['updated-url'].decode('ascii', 'replace')
  147.             self.contentType = None
  148.             
  149.             try:
  150.                 self.contentType = info['content-type'].decode('ascii', 'replace')
  151.             except:
  152.                 self.contentType = None
  153.  
  154.             self.runDownloader()
  155.         else:
  156.             error = httpclient.UnexpectedStatusCode(info['status'])
  157.             self.onContentTypeError(error)
  158.  
  159.     
  160.     def onContentTypeError(self, error):
  161.         if not self.idExists():
  162.             return None
  163.         
  164.         self.status['state'] = u'failed'
  165.         self.status['shortReasonFailed'] = error.getFriendlyDescription()
  166.         self.status['reasonFailed'] = error.getLongDescription()
  167.         self.signalChange()
  168.  
  169.     
  170.     def getContentType(self):
  171.         httpclient.grabHeaders(self.url, self.onContentType, self.onContentTypeError)
  172.  
  173.     
  174.     def initializeDaemon(cls):
  175.         RemoteDownloader.dldaemon = daemon.ControllerDaemon()
  176.  
  177.     initializeDaemon = classmethod(initializeDaemon)
  178.     
  179.     def _getRates(self):
  180.         state = self.getState()
  181.         if state == u'downloading':
  182.             return (self.status.get('rate', 0), self.status.get('upRate', 0))
  183.         
  184.         if state == u'uploading':
  185.             return (0, self.status.get('upRate', 0))
  186.         
  187.         return (0, 0)
  188.  
  189.     
  190.     def beforeChangingStatus(self):
  191.         global totalDownRate, totalUpRate
  192.         rates = self._getRates()
  193.         totalDownRate -= rates[0]
  194.         totalUpRate -= rates[1]
  195.  
  196.     
  197.     def afterChangingStatus(self):
  198.         global totalDownRate, totalUpRate
  199.         rates = self._getRates()
  200.         totalDownRate += rates[0]
  201.         totalUpRate += rates[1]
  202.  
  203.     
  204.     def updateStatus(cls, data):
  205.         for field in data:
  206.             if field not in ('filename', 'shortFilename', 'channelName', 'metainfo', 'fastResumeData'):
  207.                 data[field] = unicodify(data[field])
  208.                 continue
  209.         
  210.         self = _getDownloader(dlid = data['dlid'])
  211.         if self is not None:
  212.             
  213.             try:
  214.                 if self.status == data:
  215.                     return None
  216.             except Exception:
  217.                 e = None
  218.                 print 'WARNING exception when comparing status: %s' % e
  219.  
  220.             wasFinished = self.isFinished()
  221.             self.beforeChangingStatus()
  222.             if data.has_key('activity') and data['activity']:
  223.                 data['activity'] = _(data['activity'])
  224.             
  225.             self.status = data
  226.             if self.isFinished():
  227.                 pass
  228.             finished = not wasFinished
  229.             self.afterChangingStatus()
  230.             if self.getState() == u'uploading' and not (self.manualUpload) and self.getUploadRatio() > 1.5:
  231.                 self.stopUpload()
  232.             
  233.             self.signalChange(needsSignalItem = not finished)
  234.             if finished:
  235.                 for item in self.itemList:
  236.                     item.onDownloadFinished()
  237.                 
  238.             
  239.         
  240.  
  241.     updateStatus = classmethod(updateStatus)
  242.     
  243.     def runDownloader(self):
  244.         flashscraper.tryScrapingURL(self.url, self._runDownloader)
  245.  
  246.     
  247.     def _runDownloader(self, url, contentType = None):
  248.         if not self.idExists():
  249.             return None
  250.         
  251.         if contentType is not None:
  252.             self.contentType = contentType
  253.         
  254.         if url is not None:
  255.             self.url = url
  256.             c = command.StartNewDownloadCommand(RemoteDownloader.dldaemon, self.url, self.dlid, self.contentType, self.channelName)
  257.             c.send()
  258.             _downloads[self.dlid] = self
  259.         else:
  260.             self.status['state'] = u'failed'
  261.             self.status['shortReasonFailed'] = _('File not found')
  262.             self.status['reasonFailed'] = _('Flash URL Scraping Error')
  263.         self.signalChange()
  264.  
  265.     
  266.     def pause(self, block = False):
  267.         if _downloads.has_key(self.dlid):
  268.             c = command.PauseDownloadCommand(RemoteDownloader.dldaemon, self.dlid)
  269.             c.send()
  270.         else:
  271.             self.beforeChangingStatus()
  272.             self.status['state'] = u'paused'
  273.             self.afterChangingStatus()
  274.             self.signalChange()
  275.  
  276.     
  277.     def stop(self, delete):
  278.         if self.getState() in (u'downloading', u'uploading', u'paused'):
  279.             if _downloads.has_key(self.dlid):
  280.                 c = command.StopDownloadCommand(RemoteDownloader.dldaemon, self.dlid, delete)
  281.                 c.send()
  282.                 del _downloads[self.dlid]
  283.             
  284.         elif delete:
  285.             self.delete()
  286.         
  287.         self.status['state'] = u'stopped'
  288.         self.signalChange()
  289.  
  290.     
  291.     def delete(self):
  292.         
  293.         try:
  294.             filename = self.status['filename']
  295.         except KeyError:
  296.             return None
  297.  
  298.         
  299.         try:
  300.             fileutil.delete(filename)
  301.         except:
  302.             logging.warn('Error deleting downloaded file: %s\n%s' % (templatehelper.toUni(stringify(filename)), traceback.format_exc()))
  303.  
  304.         parent = os.path.join(filename, os.path.pardir)
  305.         parent = os.path.normpath(parent)
  306.         moviesDir = config.get(prefs.MOVIES_DIRECTORY)
  307.         if os.path.exists(parent) and os.path.exists(moviesDir) and not platformutils.samefile(parent, moviesDir) and len(os.listdir(parent)) == 0:
  308.             
  309.             try:
  310.                 os.rmdir(parent)
  311.             logging.warn('Error deleting empty download directory: %s\n%s' % (templatehelper.toUni(parent), traceback.format_exc()))
  312.  
  313.         
  314.  
  315.     
  316.     def start(self):
  317.         if self.getState() == u'failed':
  318.             if _downloads.has_key(self.dlid):
  319.                 del _downloads[self.dlid]
  320.             
  321.             self.dlid = generateDownloadID()
  322.             views.remoteDownloads.recomputeIndex(indexes.downloadsByDLID)
  323.             self.beforeChangingStatus()
  324.             self.status = { }
  325.             self.afterChangingStatus()
  326.             if self.contentType == u'':
  327.                 self.getContentType()
  328.             else:
  329.                 self.runDownloader()
  330.             self.signalChange()
  331.         elif self.getState() in (u'stopped', u'paused', u'offline'):
  332.             if _downloads.has_key(self.dlid):
  333.                 c = command.StartDownloadCommand(RemoteDownloader.dldaemon, self.dlid)
  334.                 c.send()
  335.             else:
  336.                 self.status['state'] = u'downloading'
  337.                 self.restart()
  338.                 self.signalChange()
  339.         
  340.  
  341.     
  342.     def migrate(self, directory):
  343.         if _downloads.has_key(self.dlid):
  344.             c = command.MigrateDownloadCommand(RemoteDownloader.dldaemon, self.dlid, directory)
  345.             c.send()
  346.         else:
  347.             
  348.             try:
  349.                 shortFilename = self.status['shortFilename']
  350.             except KeyError:
  351.                 print "WARNING: can't migrate download because we don't have a shortFilename!\nURL was %s" % self.url
  352.                 return None
  353.  
  354.             
  355.             try:
  356.                 filename = self.status['filename']
  357.             except KeyError:
  358.                 print "WARNING: can't migrate download because we don't have a filename!\nURL was %s" % self.url
  359.                 return None
  360.  
  361.             if os.path.exists(filename):
  362.                 if 'channelName' in self.status and self.status['channelName'] is not None:
  363.                     channelName = filterDirectoryName(self.status['channelName'])
  364.                     directory = os.path.join(directory, channelName)
  365.                 
  366.                 
  367.                 try:
  368.                     os.makedirs(directory)
  369.                 except:
  370.                     pass
  371.  
  372.                 newfilename = os.path.join(directory, shortFilename)
  373.                 if newfilename == filename:
  374.                     return None
  375.                 
  376.                 newfilename = nextFreeFilename(newfilename)
  377.                 
  378.                 def callback():
  379.                     self.status['filename'] = newfilename
  380.                     self.signalChange()
  381.  
  382.                 fileutil.migrate_file(filename, newfilename, callback)
  383.             
  384.         for i in self.itemList:
  385.             i.migrateChildren(directory)
  386.         
  387.  
  388.     
  389.     def setDeleteFiles(self, deleteFiles):
  390.         self.deleteFiles = deleteFiles
  391.  
  392.     
  393.     def setChannelName(self, channelName):
  394.         if self.channelName is None:
  395.             if channelName:
  396.                 checkF(channelName)
  397.             
  398.             self.channelName = channelName
  399.         
  400.  
  401.     
  402.     def remove(self):
  403.         global totalDownRate, totalUpRate
  404.         rates = self._getRates()
  405.         totalDownRate -= rates[0]
  406.         totalUpRate -= rates[1]
  407.         self.stop(self.deleteFiles)
  408.         DDBObject.remove(self)
  409.  
  410.     
  411.     def getType(self):
  412.         '''Get the type of download.  Will return either "http" or
  413.         "bittorrent".
  414.         '''
  415.         self.confirmDBThread()
  416.         if self.contentType == u'application/x-bittorrent':
  417.             return u'bittorrent'
  418.         else:
  419.             return u'http'
  420.  
  421.     
  422.     def addItem(self, item):
  423.         if item not in self.itemList:
  424.             self.itemList.append(item)
  425.         
  426.  
  427.     
  428.     def removeItem(self, item):
  429.         self.itemList.remove(item)
  430.         if len(self.itemList) == 0:
  431.             self.remove()
  432.         
  433.  
  434.     
  435.     def getRate(self):
  436.         self.confirmDBThread()
  437.         return self.status.get('rate', 0)
  438.  
  439.     
  440.     def getETA(self):
  441.         self.confirmDBThread()
  442.         return self.status.get('eta', 0)
  443.  
  444.     
  445.     def getStartupActivity(self):
  446.         self.confirmDBThread()
  447.         activity = self.status.get('activity')
  448.         if activity is None:
  449.             return _('starting up')
  450.         else:
  451.             return activity
  452.  
  453.     getStartupActivity = returnsUnicode(getStartupActivity)
  454.     
  455.     def getReasonFailed(self):
  456.         if not self.getState() == u'failed':
  457.             msg = u'getReasonFailed() called on a non-failed downloader'
  458.             raise ValueError(msg)
  459.         
  460.         self.confirmDBThread()
  461.         return self.status['reasonFailed']
  462.  
  463.     getReasonFailed = returnsUnicode(getReasonFailed)
  464.     
  465.     def getShortReasonFailed(self):
  466.         if not self.getState() == u'failed':
  467.             msg = u'getShortReasonFailed() called on a non-failed downloader'
  468.             raise ValueError(msg)
  469.         
  470.         self.confirmDBThread()
  471.         return self.status['shortReasonFailed']
  472.  
  473.     getShortReasonFailed = returnsUnicode(getShortReasonFailed)
  474.     
  475.     def getURL(self):
  476.         self.confirmDBThread()
  477.         return self.url
  478.  
  479.     getURL = returnsUnicode(getURL)
  480.     
  481.     def getState(self):
  482.         self.confirmDBThread()
  483.         return self.status.get('state', u'downloading')
  484.  
  485.     getState = returnsUnicode(getState)
  486.     
  487.     def isFinished(self):
  488.         return self.getState() in (u'finished', u'uploading')
  489.  
  490.     
  491.     def getTotalSize(self):
  492.         self.confirmDBThread()
  493.         return self.status.get(u'totalSize', -1)
  494.  
  495.     
  496.     def getCurrentSize(self):
  497.         self.confirmDBThread()
  498.         return self.status.get(u'currentSize', 0)
  499.  
  500.     
  501.     def getFilename(self):
  502.         self.confirmDBThread()
  503.         return self.status.get('filename', FilenameType(''))
  504.  
  505.     getFilename = returnsFilename(getFilename)
  506.     
  507.     def onRestore(self):
  508.         self.deleteFiles = True
  509.         self.itemList = []
  510.         if self.dlid == 'noid':
  511.             self.dlid = generateDownloadID()
  512.         
  513.         self.status['rate'] = 0
  514.         self.status['upRate'] = 0
  515.         self.status['eta'] = 0
  516.  
  517.     
  518.     def getUploadRatio(self):
  519.         size = self.getCurrentSize()
  520.         if size == 0:
  521.             return 0
  522.         
  523.         return self.status.get('uploaded', 0) / size
  524.  
  525.     
  526.     def restartIfNeeded(self):
  527.         if self.getState() in (u'downloading', u'offline'):
  528.             self.restart()
  529.         
  530.         if self.getState() in u'uploading':
  531.             if self.manualUpload or self.getUploadRatio() < 1.5:
  532.                 self.restart()
  533.             else:
  534.                 self.stopUpload()
  535.         
  536.  
  537.     
  538.     def restart(self):
  539.         if len(self.status) == 0 or self.status.get('dlerType') is None:
  540.             if self.contentType == u'':
  541.                 self.getContentType()
  542.             else:
  543.                 self.runDownloader()
  544.         else:
  545.             _downloads[self.dlid] = self
  546.             c = command.RestoreDownloaderCommand(RemoteDownloader.dldaemon, self.status)
  547.             c.send()
  548.  
  549.     
  550.     def startUpload(self):
  551.         if self.getState() != u'finished' or self.getType() != u'bittorrent':
  552.             return None
  553.         
  554.         self.manualUpload = True
  555.         if _downloads.has_key(self.dlid):
  556.             c = command.StartDownloadCommand(RemoteDownloader.dldaemon, self.dlid)
  557.             c.send()
  558.         else:
  559.             self.beforeChangingStatus()
  560.             self.status['state'] = u'uploading'
  561.             self.afterChangingStatus()
  562.             self.restart()
  563.             self.signalChange()
  564.  
  565.     
  566.     def stopUpload(self):
  567.         if self.getState() != u'uploading':
  568.             return None
  569.         
  570.         if _downloads.has_key(self.dlid):
  571.             c = command.StopUploadCommand(RemoteDownloader.dldaemon, self.dlid)
  572.             c.send()
  573.             del _downloads[self.dlid]
  574.         
  575.         self.beforeChangingStatus()
  576.         self.status['state'] = u'finished'
  577.         self.afterChangingStatus()
  578.         self.signalChange()
  579.  
  580.  
  581.  
  582. def cleanupIncompleteDownloads():
  583.     downloadDir = os.path.join(config.get(prefs.MOVIES_DIRECTORY), 'Incomplete Downloads')
  584.     if not os.path.exists(downloadDir):
  585.         return None
  586.     
  587.     filesInUse = set()
  588.     views.remoteDownloads.confirmDBThread()
  589.     for downloader in views.remoteDownloads:
  590.         if downloader.getState() in ('downloading', 'paused', 'offline'):
  591.             filename = downloader.getFilename()
  592.             if len(filename) > 0:
  593.                 if not os.path.isabs(filename):
  594.                     filename = os.path.join(downloadDir, filename)
  595.                 
  596.                 filesInUse.add(filename)
  597.             
  598.         len(filename) > 0
  599.     
  600.     for f in os.listdir(downloadDir):
  601.         f = os.path.join(downloadDir, f)
  602.         if f not in filesInUse:
  603.             
  604.             try:
  605.                 if os.path.isfile(f):
  606.                     os.remove(f)
  607.                 elif os.path.isdir(f):
  608.                     shutil.rmtree(f)
  609.  
  610.             continue
  611.     
  612.  
  613.  
  614. def restartDownloads():
  615.     views.remoteDownloads.confirmDBThread()
  616.     for downloader in views.remoteDownloads:
  617.         downloader.restartIfNeeded()
  618.     
  619.  
  620.  
  621. def killUploaders(*args):
  622.     torrent_limit = config.get(prefs.UPSTREAM_TORRENT_LIMIT)
  623.     while views.autoUploads.len() > torrent_limit:
  624.         views.autoUploads[0].stopUpload()
  625.  
  626.  
  627. def configChangeUploaders(key, value):
  628.     if key == prefs.UPSTREAM_TORRENT_LIMIT.key:
  629.         killUploaders()
  630.     
  631.  
  632.  
  633. def limitUploaders():
  634.     views.autoUploads.addAddCallback(killUploaders)
  635.     config.addChangeCallback(configChangeUploaders)
  636.     killUploaders()
  637.  
  638.  
  639. def startupDownloader():
  640.     '''Initialize the downloaders.
  641.  
  642.     This method currently does 2 things.  It deletes any stale files self in
  643.     Incomplete Downloads, then it restarts downloads that have been restored
  644.     from the database.  It must be called before any RemoteDownloader objects
  645.     get created.
  646.     '''
  647.     cleanupIncompleteDownloads()
  648.     RemoteDownloader.initializeDaemon()
  649.     limitUploaders()
  650.     restartDownloads()
  651.  
  652.  
  653. def shutdownDownloader(callback = None):
  654.     if hasattr(RemoteDownloader, 'dldaemon') and RemoteDownloader.dldaemon is not None:
  655.         RemoteDownloader.dldaemon.shutdownDownloaderDaemon(callback = callback)
  656.     
  657.  
  658.  
  659. def lookupDownloader(url):
  660.     return views.remoteDownloads.getItemWithIndex(indexes.downloadsByURL, url)
  661.  
  662.  
  663. def getExistingDownloaderByURL(url):
  664.     downloader = lookupDownloader(url)
  665.     return downloader
  666.  
  667.  
  668. def getExistingDownloader(item):
  669.     downloader = lookupDownloader(item.getURL())
  670.     if downloader:
  671.         downloader.addItem(item)
  672.     
  673.     return downloader
  674.  
  675.  
  676. def getDownloader(item):
  677.     existing = getExistingDownloader(item)
  678.     if existing:
  679.         return existing
  680.     
  681.     url = item.getURL()
  682.     channelName = platformutils.unicodeToFilename(item.getChannelTitle(True))
  683.     if not channelName:
  684.         channelName = None
  685.     
  686.     if url.startswith(u'file://'):
  687.         path = getFileURLPath(url)
  688.         
  689.         try:
  690.             getTorrentInfoHash(path)
  691.         except ValueError:
  692.             raise ValueError("Don't know how to handle %s" % url)
  693.         except IOError:
  694.             return None
  695.  
  696.         return RemoteDownloader(url, item, u'application/x-bittorrent', channelName = channelName)
  697.     else:
  698.         return RemoteDownloader(url, item, channelName = channelName)
  699.  
  700.